package com.sentiment.crawler;

import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Pattern;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.sentiment.contentextractor.ContentExtractor;
import com.sentiment.contentextractor.News;
import com.sentiment.parser.news.Filter;
import com.sentiment.webcollector.crawler.DeepCrawler;
import com.sentiment.webcollector.model.Links;
import com.sentiment.webcollector.model.Page;

public class SohuNewsCrawler extends DeepCrawler {
	
	private static final Logger LOG = LoggerFactory.getLogger(SohuNewsCrawler.class);
	
	private List<News> newsList = new ArrayList<News>();
	
	public List<News> getNewsList() {
		return newsList;
	}
	
	public SohuNewsCrawler(String crawlPath) {
		super(crawlPath);
		// TODO Auto-generated constructor stub
	}

	@Override
	public Links visitAndGetNextLinks(Page page) {
		// TODO Auto-generated method stub
		String url = page.getUrl();
		if (Pattern.matches("http://news.sohu.com/ns\\?word=.*", url)){
			Links links = new Links();
			links.addAllFromDocument(page.getDoc(), ".c-title");
			LOG.debug("Sohu News Engine : " + links);
			return links;
		} else {
			News n = null;
			try {
				n = ContentExtractor.getNewsByHtml(page.getHtml());
				n.setUrl(page.getUrl());
			} catch (Exception e) {
				e.printStackTrace();
			}
			if (n != null) {
				newsList.add(n);
				LOG.info(" add URL " + page.getUrl() + " to newsList success ");
			} else {
				LOG.info(" add URL " + page.getUrl() + " to newsList failed ");
			}
		}
		return null;
	}
	
	public void getSearch(String keyword) {
		try {
			addSeed("http://news.sohu.com/ns?word=" + URLEncoder.encode(keyword, "utf-8"));
			start(2);
		} catch (Exception e) {
			LOG.error(e.toString());
		}
	}
}
